﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Windows.Forms.DataVisualization;
using System.Windows.Forms.DataVisualization.Charting;
using MathNet.Numerics.Distributions;
using System.IO;
//using STATCONNECTORCLNTLib;
//using StatConnectorCommonLib;
//using STATCONNECTORSRVLib;
//using RDotNet;
//using System.Numerics;
using Extreme.Statistics;
using Extreme.Statistics.Tests;

namespace TA_Zipf_Distribution
{
    public partial class Main : Form
    {
        Boolean showStat;
        //StatConnector rconn;

        struct DataElement
        {
            public int index;
            public int score;
            public int bucketIndex;
        }

        public Main()
        {
            InitializeComponent();
            textBoxNumberOfMultipleFile.Enabled = false;
            checkBoxSaveTAResultToFile.Checked = true;
            textBoxSaveTAResultToFile.Enabled = true;
            checkBoxSaveAnytimeTAResultToFile.Checked = true;
            textBoxSaveAnytimeTAResultToFile.Enabled = true;
            radioButtonCalculatePopParams.Checked = true;
            radioButtonSampleParams.Checked = false;
            radioButtonGivenPopulationParams.Checked = false;
            checkBoxSaveCIResultToFile.Checked = true;
            caluclatingParamsTime = TimeSpan.Zero;
            checkBoxProbTACalculatePopParamsSaveToFile.Checked = true;
            showStat = true;
            checkBoxShowState.Checked = true;
            checkBoxProbTASampleParamSaveToFile.Checked = true;
            //rconn = new STATCONNECTORSRVLib.StatConnector();
            //rconn.Init("R");
        }

        const string strDataPath = "C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\";

        double sZipf;

        int numberOfList, numberOfItem;
        List<DataElement[]> lists;
        DataElement[] list;

        long[] sumList;
        int numberOfBucket;

        int[] maxValueList;
        List<int[]> histograms;
        List<double[]> rangeLists;
        double[] sumRange;
        int[] sumHistogram;
        long maxSum;
        long minSum;

        List<DataElement> TAresultLists;
        int TAminScore;
        int numberOfItemHaveBeenRead;
        int numberOfItemHaveBeenSeen;
        bool[] markDataItem;
        int valueOfK;

        double sampleSum;
        double sampleMean;
        double estimatedStandardDeviation;

        double populationSum;
        double populationMean;
        double populationStandardDeviation;

        double givenProb;

        TimeSpan caluclatingParamsTime;

        double inUsedMean;
        double inUsedStandarDeviation;

        List<DataElement> anytimeTAResultLists;
        int anytimeTAMinScore;
        double inputConfidence;
        double[] oneUnseenPDF;
        double totalCases;
        int roundUpTo;

        #region Generate data with Zipf Distribution and Save into file
        private void buttonGenerateData_Click(object sender, EventArgs e)
        {
            labelStatusGenerateData.ForeColor = Color.Black;
            labelStatusGenerateData.Text = "Running...";
            labelStatusGenerateData.Refresh();

            numberOfList = Convert.ToInt32(textBoxNumberOfList.Text);
            numberOfItem = Convert.ToInt32(textBoxNumberOfItem.Text);

            int i, j;
            var zipfData = new int[numberOfItem];
            sZipf = Double.Parse(textBoxParameterS.Text);

            StreamWriter swDataFile = new StreamWriter(strDataPath + textBoxSaveData.Text);
            swDataFile.WriteLine("{0}", numberOfList);
            swDataFile.WriteLine("{0}", numberOfItem);
            swDataFile.WriteLine("{0:0.00}", sZipf);
            for (i = 0; i < numberOfList; i++)
            {
                swDataFile.WriteLine("--------------------------------------------------");
                var zipfRNG = new Zipf(sZipf, numberOfItem);
                int seed = DateTime.Now.Year + DateTime.Now.Month + DateTime.Now.Day
                         + DateTime.Now.Hour + DateTime.Now.Minute + DateTime.Now.Second
                         + DateTime.Now.Millisecond + i;
                zipfRNG.RandomSource = new Random(seed);
                for (j = 0; j < numberOfItem; j++)
                {
                    zipfData[j] = zipfRNG.Sample();
                    swDataFile.WriteLine(zipfData[j]);
                }
            }
            swDataFile.Close();

            labelStatusGenerateData.ForeColor = Color.Green;
            labelStatusGenerateData.Text = "DONE";
            labelStatusGenerateData.Refresh();
        }
        #endregion

        private void checkBoxReadFromMultipleFiles_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBoxReadFromMultipleFiles.Checked)
                textBoxNumberOfMultipleFile.Enabled = true;
            else
                textBoxNumberOfMultipleFile.Enabled = false;
        }

        private void checkBoxSaveTAResultToFile_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBoxSaveTAResultToFile.Checked)
                textBoxSaveTAResultToFile.Enabled = true;
            else textBoxSaveTAResultToFile.Enabled = false;
        }

        private void checkBoxSaveAnytimeTAResultToFile_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBoxSaveAnytimeTAResultToFile.Checked)
                textBoxSaveAnytimeTAResultToFile.Enabled = true;
            else textBoxSaveAnytimeTAResultToFile.Enabled = false;
        }

        #region Read data from file
        private void buttonReadData_Click(object sender, EventArgs e)
        {
            labelStatusReadData.ForeColor = Color.Black;
            labelStatusReadData.Text = "Running";
            labelStatusReadData.Refresh();

            StreamReader srDataFile;
            string line;
            int i, j;

            if (!checkBoxReadFromMultipleFiles.Checked)
            {
                srDataFile = new StreamReader(strDataPath + textBoxReadData.Text);

                lists = new List<DataElement[]>();

                line = srDataFile.ReadLine();
                numberOfList = int.Parse(line);
                line = srDataFile.ReadLine();
                numberOfItem = int.Parse(line);
                line = srDataFile.ReadLine();
                //sZipf = double.Parse(line);

                maxValueList = new int[numberOfList];
                for (i = 0; i < numberOfList; i++)
                    maxValueList[i] = 1;

                sumList = new long[numberOfItem];
                maxSum = 0;
                for (i = 0; i < numberOfList; i++)
                {
                    list = new DataElement[numberOfItem];
                    line = srDataFile.ReadLine();
                    for (j = 0; j < numberOfItem; j++)
                    {
                        line = srDataFile.ReadLine();
                        list[j].index = j + 1;
                        list[j].score = int.Parse(line);
                        sumList[j] += (long)list[j].score;
                        if (sumList[j] > maxSum)
                            maxSum = sumList[j];
                        if (list[j].score > maxValueList[i])
                            maxValueList[i] = list[j].score;
                    }
                    lists.Add(list);
                }

                srDataFile.Close();
            } 
            else {
                lists = new List<DataElement[]>();

                int k;
                int numberOfMultipleFile = Convert.ToInt32(textBoxNumberOfMultipleFile.Text);

                numberOfList = 0;

                for (k = 1; k <= numberOfMultipleFile; k++)
                {
                    if (k < 10)
                        srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + "0" + k.ToString() + ".txt");
                    else
                        srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + k.ToString() + ".txt");

                    line = srDataFile.ReadLine();
                    numberOfList += int.Parse(line);
                    line = srDataFile.ReadLine();
                    numberOfItem = int.Parse(line);

                    srDataFile.Close();
                }

                maxValueList = new int[numberOfList];
                for (i = 0; i < numberOfList; i++)
                    maxValueList[i] = 1;

                sumList = new long[numberOfItem];
                maxSum = 0;

                int t = 0;
                for (k = 1; k <= numberOfMultipleFile; k++)
                {
                    if (k < 10)
                        srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + "0" + k.ToString() + ".txt");
                    else
                        srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + k.ToString() + ".txt");

                    line = srDataFile.ReadLine();
                    int n = int.Parse(line);
                    line = srDataFile.ReadLine();
                    numberOfItem = int.Parse(line);
                    line = srDataFile.ReadLine();
                    //sZipf = double.Parse(line);

                    for (i = 0; i < n; i++)
                    {
                        list = new DataElement[numberOfItem];
                        line = srDataFile.ReadLine();
                        for (j = 0; j < numberOfItem; j++)
                        {
                            line = srDataFile.ReadLine();
                            list[j].index = j + 1;
                            list[j].score = int.Parse(line);
                            sumList[j] += (long)list[j].score;
                            if (sumList[j] > maxSum)
                                maxSum = sumList[j];
                            if (list[j].score > maxValueList[t])
                                maxValueList[t] = list[j].score;
                        }
                        lists.Add(list);
                        t++;
                    }

                    srDataFile.Close();
                }
            }

            labelStatusReadData.ForeColor = Color.Green;
            labelStatusReadData.Text = "DONE";
            labelStatusReadData.Refresh();
        }
        #endregion

        #region Data Statistics and Display Chart
        private void buttonDataStatistics_Click(object sender, EventArgs e)
        {
            int i, j, k;
            double step, current;

            #region Data Statistics
            labelStatusDataStatistics.ForeColor = Color.Black;
            labelStatusDataStatistics.Text = "Running Statistics...";
            labelStatusDataStatistics.Refresh();

            numberOfBucket = int.Parse(textBoxNumberOfBucket.Text);

            minSum = sumList[0];
            for (i = 1; i < numberOfItem; i++)
                if (sumList[i] < minSum)
                    minSum = sumList[i];

            sumRange = new double[numberOfBucket];
            step = ((double)maxSum - (double)minSum) / (double)numberOfBucket;
            current = 0;
            for (i = 0; i < numberOfBucket - 1; i++)
            {
                sumRange[i] = current + step;
                current += step;
            }
            sumRange[numberOfBucket - 1] = (double)maxSum;

            sumHistogram = new int[numberOfItem];
            for (i = 0; i < numberOfItem; i++)
            {
                for (k = 0; k < numberOfBucket; k++)
                    if (k > 0)
                    {
                        if ((sumRange[k - 1] < (double)sumList[i]) && ((double)sumList[i] <= sumRange[k]))
                        {
                            sumHistogram[k] = sumHistogram[k] + 1;
                            break;
                        }
                    }
                    else
                    {
                        if (((double)(minSum) < (double)sumList[i]) && ((double)sumList[i] <= sumRange[k]))
                        {
                            sumHistogram[k] = sumHistogram[k] + 1;
                            break;
                        }
                    }
            }

            rangeLists = new List<double[]>();
            for (i = 0; i < numberOfList; i++)
            {
                double[] range = new double[numberOfBucket];
                step = (double)maxValueList[i] / (double)numberOfBucket;
                current = 0;

                for (j = 0; j < numberOfBucket - 1; j++)
                {
                    range[j] = current + step;
                    current += step;
                }

                range[numberOfBucket - 1] = (double)maxValueList[i];
                rangeLists.Add(range);
            }

            histograms = new List<int[]>();
            for (i = 0; i < numberOfList; i++)
            {
                int[] histogram = new int[numberOfBucket];
                DataElement[] list = lists[i];
                double[] range = rangeLists[i];

                for (j = 0; j < numberOfBucket; j++)
                    histogram[j] = 0;

                for (j = 0; j < numberOfItem; j++)
                {
                    for (k = 0; k < numberOfBucket; k++)
                        if (k > 0)
                        {
                            if ((range[k - 1] < (double)list[j].score) && ((double)list[j].score <= range[k]))
                            {
                                histogram[k] = histogram[k] + 1;
                                break;
                            }
                        }
                        else
                        {
                            if (((double)(0) < (double)list[j].score) && ((double)list[j].score <= range[k]))
                            {
                                histogram[k] = histogram[k] + 1;
                                break;
                            }
                        }
                }

                histograms.Add(histogram);
            }
            #endregion

            #region Display Chart
            labelStatusDataStatistics.ForeColor = Color.Black;
            labelStatusDataStatistics.Text = "Displaying Chart...";
            labelStatusDataStatistics.Refresh();

            DisplayStatisticsInfo windowStatisticInfo = new DisplayStatisticsInfo();
            windowStatisticInfo.flowLayoutPanelChart.AutoScroll = true;

            Chart chart;
            ChartArea chartArea;
            Series series;

            for (i = 0; i < numberOfList; i++)
            {
                chart = new Chart();
                chart.Titles.Add(String.Format("Histogram for List " + (i + 1).ToString() + "\nNumber of bucket = {2}\ns = {0:0.0}\nN = {1}", sZipf, numberOfItem, numberOfBucket));
                chart.Titles[0].Font = new Font("Times New Roman", 12, FontStyle.Bold);

                chart.Width = numberOfBucket * 50;
                chartArea = new ChartArea();
                chartArea.AxisX.MinorTickMark.Enabled = false;
                chartArea.AxisX.MajorGrid.Enabled = false;
                chartArea.AxisY.MinorTickMark.Enabled = false;
                chartArea.AxisY.MajorGrid.Enabled = false;
                chartArea.AxisX.Interval = 1;
                chartArea.AxisX.LabelAutoFitStyle = LabelAutoFitStyles.LabelsAngleStep90;

                chart.ChartAreas.Add(chartArea);

                series = new Series();

                for (j = 0; j < numberOfBucket; j++)
                {
                    series.Points.Add(histograms[i][j]);
                    series.Points[j].ToolTip = histograms[i][j].ToString();
                    series.Points[j].AxisLabel = String.Format("{0:#.##}", rangeLists[i][j]);
                }

                series["PixelPointWidth"] = "20";
                chart.Series.Add(series);

                windowStatisticInfo.flowLayoutPanelChart.Controls.Add(chart);
            }

            chart = new Chart();
            chart.Titles.Add(String.Format("Histogram for Sum" + "\nNumber of bucket = {0}", numberOfBucket));
            chart.Titles[0].Font = new Font("Times New Roman", 12, FontStyle.Bold);

            chart.Width = numberOfBucket * 50;
            chartArea = new ChartArea();
            chartArea.AxisX.MinorTickMark.Enabled = false;
            chartArea.AxisX.MajorGrid.Enabled = false;
            chartArea.AxisY.MinorTickMark.Enabled = false;
            chartArea.AxisY.MajorGrid.Enabled = false;
            chartArea.AxisX.Interval = 1;
            chartArea.AxisX.LabelAutoFitStyle = LabelAutoFitStyles.LabelsAngleStep90;

            chart.ChartAreas.Add(chartArea);

            series = new Series();

            for (j = 0; j < numberOfBucket; j++)
            {
                series.Points.Add(sumHistogram[j]);
                series.Points[j].ToolTip = sumHistogram[j].ToString();
                series.Points[j].AxisLabel = String.Format("{0:#.##}", sumRange[j]);
            }

            series["PixelPointWidth"] = "20";
            chart.Series.Add(series);

            windowStatisticInfo.flowLayoutPanelChart.Controls.Add(chart);
            #endregion

            labelStatusDataStatistics.ForeColor = Color.Green;
            labelStatusDataStatistics.Text = "DONE";
            labelStatusDataStatistics.Refresh();

            windowStatisticInfo.Show();
        }
        #endregion

        #region Sort Data
        private void buttonSortData_Click(object sender, EventArgs e)
        {
            labelStatusSortData.ForeColor = Color.Black;
            labelStatusSortData.Text = "Sorting...";
            labelStatusSortData.Refresh();

            int i;
            for (i = 0; i < numberOfList; i++)
            {
                List<DataElement> currentList = new List<DataElement>(lists[i]);
                currentList.Sort(delegate(DataElement de1, DataElement de2)
                {
                    int returnValue = 1;
                    if (de1.score > de2.score)
                        returnValue = -1;
                    else if (de1.score == de2.score)
                    {
                        if (de1.index > de2.index)
                            returnValue = -1;
                        else if (de1.index == de2.index)
                            returnValue = 0;
                    }
                    return returnValue;
                });
                lists[i] = currentList.ToArray();
            }

            labelStatusSortData.ForeColor = Color.Green;
            labelStatusSortData.Text = "DONE";
            labelStatusSortData.Refresh();
        }
        #endregion

        #region Perform TA
        private void buttonPerformTA_Click(object sender, EventArgs e)
        {
            labelStatusPerformTA.ForeColor = Color.Green;
            labelStatusPerformTA.Text = "Running...";
            labelStatusPerformTA.Refresh();

            labelNumberOfItemHaveBeenRead.Text = "0/" + (numberOfItem * numberOfList).ToString();
            labelNumberOfItemHaveBeenSeen.Text = "0/" + numberOfItem.ToString();

            valueOfK = Convert.ToInt32(textBoxValueOfK.Text);

            int i, j, k;
            int totalItems = numberOfItem * numberOfList;

            // mark all items as unread
            markDataItem = new bool[numberOfItem + 1];
            for (i = 1; i <= numberOfItem; i++)
                markDataItem[i] = false;

            // initialize result list
            TAresultLists = new List<DataElement>();
            TAminScore = 0;
            bool stop = false;
            numberOfItemHaveBeenRead = 0;
            numberOfItemHaveBeenSeen = 0;

            sampleSum = 0;

            DateTime startTime = DateTime.Now;
            for (i = 0; i < numberOfItem; i++)
            {
                int threshold = 0;
                //calculate threshold
                for (j = 0; j < numberOfList; j++)
                {
                    List<DataElement> currentList = new List<DataElement>(lists[j]);
                    threshold += currentList[i].score;
                }

                if ((TAminScore < threshold) || (TAresultLists.Count < valueOfK))
                {
                    // sequentiall access to each element of each list
                    for (j = 0; j < numberOfList; j++)
                    {

                        List<DataElement> currentList = new List<DataElement>(lists[j]);
                        DataElement currentDataItem = currentList[i];
                        int overallScore = currentDataItem.score;
                        numberOfItemHaveBeenRead++;

                        if (showStat)
                        {
                            labelNumberOfItemHaveBeenRead.Text = numberOfItemHaveBeenRead.ToString() + "/" + totalItems.ToString();
                            labelNumberOfItemHaveBeenRead.Refresh();
                        }

                        if (!markDataItem[currentDataItem.index]) // current data item's score is not read in all lists
                        {
                            markDataItem[currentDataItem.index] = true;
                            numberOfItemHaveBeenSeen++;

                            if (showStat)
                            {
                                labelNumberOfItemHaveBeenSeen.Text = numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString();
                                labelNumberOfItemHaveBeenSeen.Refresh();
                            }

                            // do random access to other list to retrieve its score in the local list
                            overallScore += retrieveLocalScores(currentDataItem.index, j);

                            sampleSum += (double)overallScore;
                            sampleMean = sampleSum / (double)numberOfItemHaveBeenSeen;

                            if (showStat)
                            {
                                labelCISampleMean.Text = String.Format("{0}", sampleMean);
                                labelCISampleMean.Refresh();
                            }

                            //the new item has a better score than the lowest one in the result set
                            // or there is still no result yet
                            if ((overallScore > TAminScore) || (TAresultLists.Count < valueOfK))
                            {
                                DataElement newItem = new DataElement();
                                newItem.index = currentDataItem.index;
                                newItem.score = overallScore;

                                if (TAresultLists.Count < valueOfK) // still no result
                                {
                                    k = 0;
                                    // find a correct position for new item in the list of descending result
                                    while ((k < TAresultLists.Count) && (TAresultLists[k].score > overallScore))
                                        k++;
                                    TAresultLists.Insert(k, newItem);
                                    TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                }
                                else
                                {
                                    // remove the item result having the minimum score
                                    if (TAresultLists.Count > 0)
                                        TAresultLists.RemoveAt(TAresultLists.Count - 1);
                                    bool added = false;
                                    // find an appropriate position for the new item result
                                    for (k = 0; k < TAresultLists.Count; k++)
                                        if (overallScore > TAresultLists[k].score)
                                        {
                                            added = true;
                                            TAresultLists.Insert(k, newItem);
                                            break;
                                        }
                                    // add to the result lsit
                                    if (!added)
                                        TAresultLists.Insert(k, newItem);

                                    // update minimum score
                                    TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                }
                            }
                        }

                        if (showStat)
                        {
                            labelTACurrentMinScore.Text = TAminScore.ToString();
                            labelTACurrentMinScore.Refresh();
                        }
                    }
                }
                else stop = true;

                if (stop)
                    break;
            }
            TimeSpan totalTime = DateTime.Now - startTime;

            if (checkBoxSaveTAResultToFile.Checked)
            {
                StreamWriter swTAResult = new StreamWriter("C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\" + textBoxSaveTAResultToFile.Text + ".ta.sorted.result.txt");
                swTAResult.WriteLine(numberOfList + " - Number of list");
                swTAResult.WriteLine(numberOfItem + " - Number of item per list");
                swTAResult.WriteLine("Zipf - Distribution of each list");
                swTAResult.WriteLine(sZipf + " - s parameter");
                swTAResult.WriteLine(valueOfK + " - Value of K");
                swTAResult.WriteLine("Sorted - Type of data");
                swTAResult.WriteLine("Traditional TA - Method");
                swTAResult.WriteLine(numberOfItemHaveBeenRead.ToString() + "/" + (numberOfItem * numberOfList).ToString() + " - Number of data read/Total data") ;
                swTAResult.WriteLine(numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString() + " - Number of items seen/Total item");
                swTAResult.WriteLine(totalTime.ToString() + " - Total run time");
                swTAResult.WriteLine("---------------------------------------");
                for (i = 0; i < valueOfK; i++)
                    swTAResult.WriteLine(String.Format("{0,8}{1,15}", TAresultLists[i].index, TAresultLists[i].score));

                swTAResult.Close();
            }

            labelStatusPerformTA.ForeColor = Color.Green;
            labelStatusPerformTA.Text = "DONE";
            labelStatusPerformTA.Refresh();
        }

        /// <summary>
        /// This function retrieves the total score of the item having itemIndex
        /// in the lists other than listIndex
        /// </summary>
        /// <param name="itemIndex"></param>
        /// <param name="listIndex"></param>
        /// <returns></returns>
        int retrieveLocalScores(int itemIndex, int listIndex)
        {
            int score = 0;
            int i, j;

            for (i = 0; i < numberOfList; i++)
                if (i != listIndex)
                {
                    int localScore = 0;
                    List<DataElement> currentList = new List<DataElement>(lists[i]);
                    for (j = 0; j < numberOfItem; j++)
                    {
                        if (currentList[j].index == itemIndex)
                        {
                            localScore = currentList[j].score;
                            break;
                        }
                    }
                    score += localScore;
                }

            return score;
        }
        #endregion

        #region Autorun
        private void buttonAutorun_Click(object sender, EventArgs e)
        {
            int totalTest = 9;
            string[] dataFrom = new string[] { 
                "100000_05_1_5.data", "100000_05_1_5.data", "100000_05_1_5.data",
                "100000_05_1_5.data", "100000_05_1_5.data", "100000_05_1_5.data",
                "100000_05_1_5.data", "100000_05_1_5.data", "100000_05_1_5.data"
            };
            string[] strNumberOfMultipleFile = new string[] { 
                "4", "5", "6",
                "4", "5", "6",
                "4", "5", "6"
            };
            string[] strValueOfK = new string[] { 
                "100", "100", "100",
                "100", "100", "100",
                "100", "100", "100"
            };
            string[] saveToFile = new string[] {
                "100000_20_1_5_0100.sorted.ta.result.txt", "100000_20_1_5_0200.sorted.ta.result.txt", "100000_20_1_5_0300.ta.sorted.result.txt",
                "100000_25_1_5_0100.sorted.ta.result.txt", "100000_25_1_5_0200.sorted.ta.result.txt", "100000_25_1_5_0300.ta.sorted.result.txt",
                "100000_30_1_5_0100.sorted.ta.result.txt", "100000_30_1_5_0200.sorted.ta.result.txt", "100000_30_1_5_0300.ta.sorted.result.txt"
            };

            /* DONE
            int totalTest = 10;
            string[] dataFrom = new string[] { "100000_05_0_5.data", 
                "100000_05_1_0.data", "100000_05_1_0.data", "100000_05_1_0.data",
                "100000_05_1_0.data", "100000_05_1_0.data", "100000_05_1_0.data",
                "100000_05_1_0.data", "100000_05_1_0.data", "100000_05_1_0.data"
            };
            string[] strNumberOfMultipleFile = new string[] { "6",
                "4", "4", "4",
                "5", "5", "5",
                "6", "6", "6"
            };
            string[] strValueOfK = new string[] { "300",
                "100", "200", "300",
                "100", "200", "300",
                "100", "200", "300"
            };
            string[] saveToFile = new string[] { "100000_30_0_5_0300.sorted.result.txt",
                "100000_20_1_0_0100.sorted.result.txt", "100000_20_1_0_0200.sorted.result.txt", "100000_20_1_0_0300.sorted.result.txt",
                "100000_25_1_0_0100.sorted.result.txt", "100000_25_1_0_0200.sorted.result.txt", "100000_25_1_0_0300.sorted.result.txt",
                "100000_30_1_0_0100.sorted.result.txt", "100000_30_1_0_0200.sorted.result.txt", "100000_30_1_0_0300.sorted.result.txt"
            };
            */

            /*
            int totalTest = 3;
            string[] dataFrom = new string[] {
                "001000_05_1_0.data", "001000_05_1_0.data", "001000_05_1_0.data"
            };
            string[] strNumberOfMultipleFile = new string[] {
                "4", "5", "6"
            };
            string[] strValueOfK = new string[] {
                "100", "200", "300"
            };
            string[] saveToFile = new string[] { 
                "001000_20_1_0_0100.sorted.result.txt", "001000_25_1_0_0200.sorted.result.txt", "001000_30_1_0_0300.sorted.result.txt"
            };
            */

            for (int ntest = 0; ntest < totalTest; ntest++)
            {
                textBoxReadData.Text = dataFrom[ntest];
                textBoxReadData.Refresh();
                checkBoxReadFromMultipleFiles.Checked = true;
                checkBoxReadFromMultipleFiles.Refresh();
                textBoxNumberOfMultipleFile.Text = strNumberOfMultipleFile[ntest];
                textBoxNumberOfMultipleFile.Refresh();
                textBoxValueOfK.Text = strValueOfK[ntest];
                textBoxValueOfK.Refresh();
                checkBoxSaveTAResultToFile.Checked = true;
                checkBoxSaveTAResultToFile.Refresh();
                textBoxSaveTAResultToFile.Text = saveToFile[ntest];
                textBoxSaveTAResultToFile.Refresh();

                #region Read Data
                labelStatusReadData.ForeColor = Color.Black;
                labelStatusReadData.Text = "Running";
                labelStatusReadData.Refresh();

                StreamReader srDataFile;
                string line;
                int i, j;

                if (!checkBoxReadFromMultipleFiles.Checked)
                {
                    srDataFile = new StreamReader(strDataPath + textBoxReadData.Text);

                    lists = new List<DataElement[]>();

                    line = srDataFile.ReadLine();
                    numberOfList = int.Parse(line);
                    line = srDataFile.ReadLine();
                    numberOfItem = int.Parse(line);
                    line = srDataFile.ReadLine();
                    sZipf = double.Parse(line);

                    maxValueList = new int[numberOfList];
                    for (i = 0; i < numberOfList; i++)
                        maxValueList[i] = 1;

                    sumList = new long[numberOfItem];
                    maxSum = 0;
                    for (i = 0; i < numberOfList; i++)
                    {
                        list = new DataElement[numberOfItem];
                        line = srDataFile.ReadLine();
                        for (j = 0; j < numberOfItem; j++)
                        {
                            line = srDataFile.ReadLine();
                            list[j].index = j + 1;
                            list[j].score = int.Parse(line);
                            sumList[j] += (long)list[j].score;
                            if (sumList[j] > maxSum)
                                maxSum = sumList[j];
                            if (list[j].score > maxValueList[i])
                                maxValueList[i] = list[j].score;
                        }
                        lists.Add(list);
                    }

                    srDataFile.Close();
                }
                else
                {
                    lists = new List<DataElement[]>();

                    int k;
                    int numberOfMultipleFile = Convert.ToInt32(textBoxNumberOfMultipleFile.Text);

                    numberOfList = 0;

                    for (k = 1; k <= numberOfMultipleFile; k++)
                    {
                        if (k < 10)
                            srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + "0" + k.ToString() + ".txt");
                        else
                            srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + k.ToString() + ".txt");

                        line = srDataFile.ReadLine();
                        numberOfList += int.Parse(line);
                        line = srDataFile.ReadLine();
                        numberOfItem = int.Parse(line);

                        srDataFile.Close();
                    }

                    maxValueList = new int[numberOfList];
                    for (i = 0; i < numberOfList; i++)
                        maxValueList[i] = 1;

                    sumList = new long[numberOfItem];
                    maxSum = 0;

                    int t = 0;
                    for (k = 1; k <= numberOfMultipleFile; k++)
                    {
                        if (k < 10)
                            srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + "0" + k.ToString() + ".txt");
                        else
                            srDataFile = new StreamReader(strDataPath + textBoxReadData.Text + k.ToString() + ".txt");

                        line = srDataFile.ReadLine();
                        int n = int.Parse(line);
                        line = srDataFile.ReadLine();
                        numberOfItem = int.Parse(line);
                        line = srDataFile.ReadLine();

                        for (i = 0; i < n; i++)
                        {
                            list = new DataElement[numberOfItem];
                            line = srDataFile.ReadLine();
                            for (j = 0; j < numberOfItem; j++)
                            {
                                line = srDataFile.ReadLine();
                                list[j].index = j + 1;
                                list[j].score = int.Parse(line);
                                sumList[j] += (long)list[j].score;
                                if (sumList[j] > maxSum)
                                    maxSum = sumList[j];
                                if (list[j].score > maxValueList[t])
                                    maxValueList[t] = list[j].score;
                            }
                            lists.Add(list);
                            t++;
                        }

                        srDataFile.Close();
                    }
                }

                labelStatusReadData.ForeColor = Color.Green;
                labelStatusReadData.Text = "DONE";
                labelStatusReadData.Refresh();
                #endregion 

                #region Sort Data
                labelStatusSortData.ForeColor = Color.Black;
                labelStatusSortData.Text = "Sorting...";
                labelStatusSortData.Refresh();

                for (i = 0; i < numberOfList; i++)
                {
                    List<DataElement> currentList = new List<DataElement>(lists[i]);
                    currentList.Sort(delegate(DataElement de1, DataElement de2)
                    {
                        int returnValue = 1;
                        if (de1.score > de2.score)
                            returnValue = -1;
                        else if (de1.score == de2.score)
                        {
                            if (de1.index > de2.index)
                                returnValue = -1;
                            else if (de1.index == de2.index)
                                returnValue = 0;
                        }
                        return returnValue;
                    });
                    lists[i] = currentList.ToArray();
                }

                labelStatusSortData.ForeColor = Color.Green;
                labelStatusSortData.Text = "DONE";
                labelStatusSortData.Refresh();
                #endregion

                #region Perform TA
                labelStatusPerformTA.ForeColor = Color.Black;
                labelStatusPerformTA.Text = "Running...";
                labelStatusPerformTA.Refresh();

                labelNumberOfItemHaveBeenRead.Text = "0/" + (numberOfItem * numberOfList).ToString();
                labelNumberOfItemHaveBeenSeen.Text = "0/" + numberOfItem.ToString();

                valueOfK = Convert.ToInt32(textBoxValueOfK.Text);

                int totalItems = numberOfItem * numberOfList;

                // mark all items as unread
                markDataItem = new bool[numberOfItem + 1];
                for (i = 1; i <= numberOfItem; i++)
                    markDataItem[i] = false;

                // initialize result list
                TAresultLists = new List<DataElement>();
                TAminScore = 0;
                bool stop = false;
                numberOfItemHaveBeenRead = 0;
                numberOfItemHaveBeenSeen = 0;

                DateTime startTime = DateTime.Now;
                for (i = 0; i < numberOfItem; i++)
                {
                    int threshold = 0;
                    //calculate threshold
                    for (j = 0; j < numberOfList; j++)
                    {
                        List<DataElement> currentList = new List<DataElement>(lists[j]);
                        threshold += currentList[i].score;
                    }

                    if ((TAminScore < threshold) || (TAresultLists.Count < valueOfK))
                    {
                        // sequentiall access to each element of each list
                        for (j = 0; j < numberOfList; j++)
                        {

                            List<DataElement> currentList = new List<DataElement>(lists[j]);
                            DataElement currentDataItem = currentList[i];
                            int overallScore = currentDataItem.score;
                            numberOfItemHaveBeenRead++;
                            labelNumberOfItemHaveBeenRead.Text = numberOfItemHaveBeenRead.ToString() + "/" + totalItems.ToString();
                            labelNumberOfItemHaveBeenRead.Refresh();

                            if (!markDataItem[currentDataItem.index]) // current data item's score is not read in all lists
                            {
                                markDataItem[currentDataItem.index] = true;
                                numberOfItemHaveBeenSeen++;
                                labelNumberOfItemHaveBeenSeen.Text = numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString();
                                labelNumberOfItemHaveBeenSeen.Refresh();

                                // do random access to other list to retrieve its score in the local list
                                overallScore += retrieveLocalScores(currentDataItem.index, j);

                                //the new item has a better score than the lowest one in the result set
                                // or there is still no result yet
                                if ((overallScore > TAminScore) || (TAresultLists.Count < valueOfK))
                                {
                                    DataElement newItem = new DataElement();
                                    newItem.index = currentDataItem.index;
                                    newItem.score = overallScore;

                                    int k;
                                    if (TAresultLists.Count < valueOfK) // still no result
                                    {
                                        k = 0;
                                        // find a correct position for new item in the list of descending result
                                        while ((k < TAresultLists.Count) && (TAresultLists[k].score > overallScore))
                                            k++;
                                        TAresultLists.Insert(k, newItem);
                                        TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                    }
                                    else
                                    {
                                        // remove the item result having the minimum score
                                        if (TAresultLists.Count > 0)
                                            TAresultLists.RemoveAt(TAresultLists.Count - 1);
                                        bool added = false;
                                        // find an appropriate position for the new item result
                                        for (k = 0; k < TAresultLists.Count; k++)
                                            if (overallScore > TAresultLists[k].score)
                                            {
                                                added = true;
                                                TAresultLists.Insert(k, newItem);
                                                break;
                                            }
                                        // add to the result lsit
                                        if (!added)
                                            TAresultLists.Insert(k, newItem);

                                        // update minimum score
                                        TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                    }
                                }
                            }
                        }
                    }
                    else stop = true;

                    if (stop)
                        break;
                }
                TimeSpan totalTime = DateTime.Now - startTime;

                StreamWriter swTAResult = new StreamWriter("C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\" + textBoxSaveTAResultToFile.Text);
                swTAResult.WriteLine(valueOfK);
                swTAResult.WriteLine(numberOfItemHaveBeenRead.ToString() + "/" + (numberOfItem * numberOfList).ToString());
                swTAResult.WriteLine(numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString());
                swTAResult.WriteLine(totalTime.ToString());
                swTAResult.WriteLine("---------------------------------------");
                for (i = 0; i < valueOfK; i++)
                    swTAResult.WriteLine(String.Format("{0,8}{1,15}", TAresultLists[i].index, TAresultLists[i].score));

                swTAResult.Close();

                labelStatusPerformTA.ForeColor = Color.Green;
                labelStatusPerformTA.Text = "DONE";
                labelStatusPerformTA.Refresh();
                #endregion
            }
        }
        #endregion

        #region Perform Anytime TA
        private void buttonPerformAnytimeTA_Click(object sender, EventArgs e)
        {
            int i, j, k;

            DateTime startTime = DateTime.Now;

            roundUpTo = Convert.ToInt32(textBoxAnytimeTARoundUpTo.Text);

            #region Building histograms
            labelAnytimeTAStatus.ForeColor = Color.Green;
            labelAnytimeTAStatus.Text = "Building histograms....";
            labelAnytimeTAStatus.Refresh();

            double step, current;

            numberOfBucket = int.Parse(textBoxAnytimeTANumberOfBucket.Text);

            double[] range = new double[numberOfBucket];

            int maxValue = maxValueList[0];
            for (i = 1; i < numberOfList; i++)
                if (maxValueList[i] > maxValue)
                    maxValue = maxValueList[i];

            step = (double)maxValue / (double)numberOfBucket;
            current = 0;
            for (j = 0; j < numberOfBucket - 1; j++)
            {
                range[j] = current + step;
                current += step;
            }
            range[numberOfBucket - 1] = (double)maxValue;

            histograms = new List<int[]>();
            for (i = 0; i < numberOfList; i++)
            {
                int[] histogram = new int[numberOfBucket];
                DataElement[] list = lists[i];

                for (j = 0; j < numberOfBucket; j++)
                    histogram[j] = 0;

                for (j = 0; j < numberOfItem; j++)
                {
                    for (k = 0; k < numberOfBucket; k++)
                        if (k > 0)
                        {
                            if ((range[k - 1] < (double)list[j].score) && ((double)list[j].score <= range[k]))
                            {
                                histogram[k] = histogram[k] + 1;
                                lists[i][j].bucketIndex = k;
                                break;
                            }
                        }
                        else
                        {
                            if (((double)(0) < (double)list[j].score) && ((double)list[j].score <= range[k]))
                            {
                                histogram[k] = histogram[k] + 1;
                                lists[i][j].bucketIndex = k;
                                break;
                            }
                        }
                }

                histograms.Add(histogram);
            }
            #endregion

            #region Performing AnytimeTA
            labelAnytimeTAStatus.Text = "Performing Anytime TA....";
            labelAnytimeTAStatus.Refresh();

            double confidence;

            valueOfK = Convert.ToInt32(textBoxAnytimeTAValueOfK.Text);
            inputConfidence = Convert.ToDouble(textBoxAnytimeTAConfidence.Text);

            current = Math.Round(step * (double)numberOfList, 3);
            int sumRangeLength = ((int)Math.Round(((double)maxValue * (double)numberOfList - current) / Math.Round(step, 2))) + 1;
            sumRange = new double[sumRangeLength];
            for (i = 0; i < sumRangeLength - 1; i++)
            {
                sumRange[i] = current;
                current = Math.Round(current + step, 3);
            }
            sumRange[sumRangeLength - 1] = (double)maxValue * (double)numberOfList;

            // mark all items as unread
            markDataItem = new bool[numberOfItem + 1];
            for (i = 1; i <= numberOfItem; i++)
                markDataItem[i] = false;

            // initialize result list
            anytimeTAResultLists = new List<DataElement>();
            anytimeTAMinScore = 0;
            //bool stop = false;
            numberOfItemHaveBeenRead = 0;
            numberOfItemHaveBeenSeen = 0;
            confidence = 0;

            bool stop = false;
            for (i = 0; i < numberOfItem; i++)
            {
                int threshold = 0;
                //calculate threshold
                for (j = 0; j < numberOfList; j++)
                {
                    DataElement[] currentList = lists[j];
                    threshold += currentList[i].score;
                }

                // sequentiall access to each element of each list
                for (j = 0; j < numberOfList; j++)
                {

                    DataElement[] currentList = lists[j];
                    DataElement currentDataItem = currentList[i];
                    int overallScore = currentDataItem.score;
                    numberOfItemHaveBeenRead++;

                    if (showStat)
                    {
                        labelAnytimeTANumberOfItemHaveBeenRead.Text = numberOfItemHaveBeenRead.ToString() + "/" + (numberOfItem * numberOfList).ToString();
                        labelAnytimeTANumberOfItemHaveBeenRead.Refresh();
                    }

                    #region Update gPDF
                    histograms[j][currentDataItem.bucketIndex]--;
                    #endregion

                    if (!markDataItem[currentDataItem.index]) // current data item's score is not read in all lists
                    {
                        markDataItem[currentDataItem.index] = true;
                        numberOfItemHaveBeenSeen++;

                        if (showStat)
                        {
                            labelAnytimeTANumberOfHaveBeenSeen.Text = numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString();
                            labelAnytimeTANumberOfHaveBeenSeen.Refresh();
                        }

                        // do random access to other list to retrieve its score in the local list
                        overallScore += retrieveLocalScores(currentDataItem.index, j);

                        //the new item has a better score than the lowest one in the result set
                        // or there is still no result yet
                        if ((overallScore > anytimeTAMinScore) || (anytimeTAResultLists.Count < valueOfK))
                        {
                            DataElement newItem = new DataElement();
                            newItem.index = currentDataItem.index;
                            newItem.score = overallScore;

                            if (anytimeTAResultLists.Count < valueOfK) // still no result
                            {
                                k = 0;
                                // find a correct position for new item in the list of descending result
                                while ((k < anytimeTAResultLists.Count) && (anytimeTAResultLists[k].score > overallScore))
                                    k++;
                                anytimeTAResultLists.Insert(k, newItem);
                                anytimeTAMinScore = anytimeTAResultLists[anytimeTAResultLists.Count - 1].score;
                            }
                            else
                            {
                                // remove the item result having the minimum score
                                if (anytimeTAResultLists.Count > 0)
                                    anytimeTAResultLists.RemoveAt(anytimeTAResultLists.Count - 1);
                                bool added = false;
                                // find an appropriate position for the new item result
                                for (k = 0; k < anytimeTAResultLists.Count; k++)
                                    if (overallScore > anytimeTAResultLists[k].score)
                                    {
                                        added = true;
                                        anytimeTAResultLists.Insert(k, newItem);
                                        break;
                                    }
                                // add to the result lsit
                                if (!added)
                                    anytimeTAResultLists.Insert(k, newItem);

                                // update minimum score
                                anytimeTAMinScore = anytimeTAResultLists[anytimeTAResultLists.Count - 1].score;
                            }
                        }
                    }

                    #region Compute Confidence 

                    #region Compute OneUnseenPDF
                    oneUnseenPDF = new double[sumRangeLength];

                    totalCases = 0;                    
                    List<double> convolution = new List<double>();

                    int x, y, z;
                    List<double> tmpConvolution = new List<double>();
                    for (x = 0; x < numberOfBucket; x++)
                        tmpConvolution.Add(histograms[0][x]);
                    convolution = new List<double>(tmpConvolution);

                    for (x = 1; x < numberOfList; x++)
                    {
                        z = -1;
                        tmpConvolution = new List<double>();
                        for (y = 0; y < convolution.Count; y++)
                        {
                            z++;
                            for (int t = 0; t < numberOfBucket; t++)
                            {
                                double tmp = convolution[y] * histograms[x][t];
                                if (z + t >= tmpConvolution.Count - 1)
                                    tmpConvolution.Add(0.0);
                                tmpConvolution[z + t] = tmpConvolution[z + t] + tmp;
                            }
                        }
                        convolution = new List<double>(tmpConvolution);
                    }

                    for (x = 0; x < sumRangeLength; x++)
                    {
                        totalCases += convolution[x];
                        oneUnseenPDF[x] = convolution[x];
                    }
                    #endregion // Compute OneUnseenPDF

                    #region Compute MaxUnseenPDF
                    k = 0;
                    confidence = 0;
                    while ((k < sumRangeLength) && ((double)anytimeTAMinScore >= (sumRange[k])))
                    {
                        confidence += oneUnseenPDF[k];
                        k++;
                    }
                    if (k < sumRangeLength)
                        confidence += oneUnseenPDF[k];

                    confidence = Math.Pow((confidence / totalCases), (double)(numberOfItem - numberOfItemHaveBeenSeen)) * 100;
                    #endregion // Compute MaxUnseenPDF

                    if (showStat)
                    {
                        labelAnytimeTACurrentMinScore.Text = anytimeTAMinScore.ToString();
                        labelAnytimeTACurrentMinScore.Refresh();
                        if (Math.Round(confidence, 2) > 0.0)
                            labelAnytimeTACurrentConfidence.Text = String.Format("{0:#.#####}%", Math.Round(confidence, 5));
                        else
                            labelAnytimeTACurrentConfidence.Text = String.Format("0.00%");
                        labelAnytimeTACurrentConfidence.Refresh();
                    }

                    if (roundUpTo > 0)
                    {
                        if ((Math.Round(confidence, roundUpTo) >= inputConfidence) && (anytimeTAResultLists.Count == valueOfK))
                        {
                            stop = true;
                            break;
                        }
                    }
                    else
                    {
                        if ((confidence >= inputConfidence) && (anytimeTAResultLists.Count == valueOfK))
                        {
                            stop = true;
                            break;
                        }
                    }

                    #endregion // Compute Confidence
                }

                if (stop)
                    break;
            }
            TimeSpan totalTime = DateTime.Now - startTime;

            if (checkBoxSaveAnytimeTAResultToFile.Checked)
            {
                StreamWriter swTAResult = new StreamWriter("C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\" + textBoxSaveAnytimeTAResultToFile.Text + ".anytimeta.sorted.result.txt");
                swTAResult.WriteLine(numberOfList + " - Number of list");
                swTAResult.WriteLine(numberOfItem + " - Number of item per list");
                swTAResult.WriteLine("Zipf - Distribution of each list");
                swTAResult.WriteLine(sZipf + " - s parameter");
                swTAResult.WriteLine(valueOfK + " - Value of K");
                swTAResult.WriteLine("Sorted - Type of data");
                swTAResult.WriteLine("Anytime TA - Method");
                swTAResult.WriteLine(numberOfBucket + " - Number of bucket");
                swTAResult.WriteLine(confidence + " - Confidence");
                //swTAResult.WriteLine(roundUpTo + " - Precision");
                swTAResult.WriteLine(numberOfItemHaveBeenRead.ToString() + "/" + (numberOfItem * numberOfList).ToString() + " - Number of data read/Total data");
                swTAResult.WriteLine(numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString() + " - Number of items seen/Total item");
                swTAResult.WriteLine(totalTime.ToString() + " - Total run time");
                swTAResult.WriteLine("---------------------------------------");
                for (i = 0; i < valueOfK; i++)
                    swTAResult.WriteLine(String.Format("{0,8}{1,15}", anytimeTAResultLists[i].index, anytimeTAResultLists[i].score));

                swTAResult.Close();
            }

            labelAnytimeTAStatus.ForeColor = Color.Green;
            labelAnytimeTAStatus.Text = "DONE";
            labelAnytimeTAStatus.Refresh();
            #endregion
        }
        #endregion

        #region Calculate Population Mean and Population Standard Deviation
        private void buttonCIPrecalculate_Click(object sender, EventArgs e)
        {
            labelCIStatus.ForeColor = Color.Green;

            labelCIStatus.Text = "Calculating Populatoin Mean...";
            labelCIStatus.Refresh();
            populationSum = 0;
            int i;

            if (showStat)
            {
                labelCaculatingPopParamsStatus.Text = "";
                labelCaculatingPopParamsStatus.Refresh();
            }
            DateTime startTime = DateTime.Now;
            for (i = 0; i < numberOfItem; i++)
            {
                if (showStat)
                {
                    labelCaculatingPopParamsStatus.Text = i.ToString();
                    labelCaculatingPopParamsStatus.Refresh();
                }

                populationSum += (double)totalScore(i);
            }

            populationMean = populationSum / (double)numberOfItem;
            inUsedMean = populationMean;

            double currentTotalScore;
            double currentSquare;
            double currentSquareSum = 0;
            labelCIStatus.Text = "Calculating Standard Deviation...";
            labelCIStatus.Refresh();
            for (i = 0; i < numberOfItem; i++)
            {
                if (showStat)
                {
                    labelCaculatingPopParamsStatus.Text = i.ToString();
                    labelCaculatingPopParamsStatus.Refresh();
                }

                currentTotalScore = (double)totalScore(i);
                currentSquare = (currentTotalScore - populationMean) * (currentTotalScore - populationMean);
                currentSquareSum += currentSquare;
            }
            populationStandardDeviation = Math.Sqrt(currentSquareSum / (double) numberOfItem);
            inUsedStandarDeviation = populationStandardDeviation;

            caluclatingParamsTime = DateTime.Now - startTime;

            if (checkBoxProbTACalculatePopParamsSaveToFile.Checked)
            {
                StreamWriter swProbTACalculatingResult = new StreamWriter("C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\" + textBoxProbTACalculatePopParamsSaveToFile.Text + ".probta.pram.txt");
                swProbTACalculatingResult.WriteLine(numberOfList + " - Number of list");
                swProbTACalculatingResult.WriteLine(numberOfItem + " - Number of item per list");
                swProbTACalculatingResult.WriteLine("Zipf - Distribution of each list");
                swProbTACalculatingResult.WriteLine(sZipf + " - s parameter");
                swProbTACalculatingResult.WriteLine("Population Mean: " + String.Format("{0}", populationMean));
                swProbTACalculatingResult.WriteLine("Population Standard Deviation: " + String.Format("{0}", populationStandardDeviation));
                swProbTACalculatingResult.WriteLine("Calculating time: " + caluclatingParamsTime.ToString());
                swProbTACalculatingResult.Close();
            }

            textBoxPopulationMean.Text = String.Format("{0}", populationMean);
            textBoxPopulationStandardDeviation.Text = String.Format("{0}", populationStandardDeviation);

            labelCIStatus.Text = "DONE";
            labelCIStatus.Refresh();

            labelCaculatingPopParamsStatus.Text = caluclatingParamsTime.ToString();
            labelCaculatingPopParamsStatus.Refresh();


        }

        /// <summary>
        /// This function retrieves the total score of the item having itemIndex
        /// </summary>
        /// <param name="itemIndex"></param>
        /// <param name="listIndex"></param>
        /// <returns></returns>
        int totalScore(int itemIndex)
        {
            int score = 0;
            int i, j;

            for (i = 0; i < numberOfList; i++)
            {
                int localScore = 0;
                List<DataElement> currentList = new List<DataElement>(lists[i]);
                for (j = 0; j < numberOfItem; j++)
                {
                    if (currentList[j].index == itemIndex)
                    {
                        localScore = currentList[j].score;
                        break;
                    }
                }
                score += localScore;
            }

            return score;
        }
        #endregion

        #region Perform Probability TA
        private void buttonPerformCITA_Click(object sender, EventArgs e)
        {
            labelCIStatus.ForeColor = Color.Green;
            labelCIStatus.Text = "Loading Mathematics library...";
            labelCIStatus.Refresh();

            Normal normDistribution = new Normal();

            if (radioButtonGivenPopulationParams.Checked)
            {
                inUsedMean = Convert.ToDouble(textBoxPopulationMean.Text);
                inUsedStandarDeviation = Convert.ToDouble(textBoxPopulationStandardDeviation.Text);
            }

            valueOfK = Convert.ToInt32(textBoxCIValueOfK.Text);

            double confidence = Convert.ToDouble(textBoxProbTAConfidence.Text);
            givenProb = calculateProbTAConfidence(confidence);
            roundUpTo = Convert.ToInt32(textBoxProbTARoundUpTo.Text);

            labelProbTACalculatedCumulativeProb.Text = String.Format("{0}", givenProb);
            labelProbTACalculatedCumulativeProb.Refresh();

            labelCIStatus.Text = "Performing Probability TA...";
            labelCIStatus.Refresh();

            if (showStat)
            {
                labelCINumberOfItemsRead.Text = "0/" + (numberOfItem * numberOfList).ToString();
                labelCINumberOfItemsSeen.Text = "0/" + numberOfItem.ToString();
            }

            int i, j, k;
            int totalItems = numberOfItem * numberOfList;

            // mark all items as unread
            markDataItem = new bool[numberOfItem + 1];
            for (i = 1; i <= numberOfItem; i++)
                markDataItem[i] = false;

            // initialize result list
            TAresultLists = new List<DataElement>();
            TAminScore = 0;
            bool stop = false;
            numberOfItemHaveBeenRead = 0;
            numberOfItemHaveBeenSeen = 0;

            sampleSum = 0;

            double zScore;
            double cumulativeProb = 0;
            double righttailProb;
            DateTime startTime = DateTime.Now;
            for (i = 0; i < numberOfItem; i++)
            {
                int threshold = 0;
                //calculate threshold
                for (j = 0; j < numberOfList; j++)
                {
                    List<DataElement> currentList = new List<DataElement>(lists[j]);
                    threshold += currentList[i].score;
                }

                if ((TAminScore < threshold) || (TAresultLists.Count < valueOfK))
                {
                    // sequentiall access to each element of each list
                    for (j = 0; j < numberOfList; j++)
                    {

                        List<DataElement> currentList = new List<DataElement>(lists[j]);
                        DataElement currentDataItem = currentList[i];
                        int overallScore = currentDataItem.score;
                        numberOfItemHaveBeenRead++;

                        if (showStat)
                        {
                            labelCINumberOfItemsRead.Text = numberOfItemHaveBeenRead.ToString() + "/" + totalItems.ToString();
                            labelCINumberOfItemsRead.Refresh();
                        }

                        if (!markDataItem[currentDataItem.index]) // current data item's score is not read in all lists
                        {
                            markDataItem[currentDataItem.index] = true;
                            numberOfItemHaveBeenSeen++;
                            if (showStat)
                            {
                                labelCINumberOfItemsSeen.Text = numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString();
                                labelCINumberOfItemsSeen.Refresh();
                            }

                            // do random access to other list to retrieve its score in the local list
                            overallScore += retrieveLocalScores(currentDataItem.index, j);

                            sampleSum += (double)overallScore;
                            sampleMean = sampleSum / (double)numberOfItemHaveBeenSeen;
                            if (showStat)
                            {
                                labelCISampleMean.Text = String.Format("{0}", sampleMean);
                                labelCISampleMean.Refresh();
                            }

                            //the new item has a better score than the lowest one in the result set
                            // or there is still no result yet
                            if ((overallScore > TAminScore) || (TAresultLists.Count < valueOfK))
                            {
                                DataElement newItem = new DataElement();
                                newItem.index = currentDataItem.index;
                                newItem.score = overallScore;

                                if (TAresultLists.Count < valueOfK) // still no result
                                {
                                    k = 0;
                                    // find a correct position for new item in the list of descending result
                                    while ((k < TAresultLists.Count) && (TAresultLists[k].score > overallScore))
                                        k++;
                                    TAresultLists.Insert(k, newItem);
                                    TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                }
                                else
                                {
                                    // remove the item result having the minimum score
                                    if (TAresultLists.Count > 0)
                                        TAresultLists.RemoveAt(TAresultLists.Count - 1);
                                    bool added = false;
                                    // find an appropriate position for the new item result
                                    for (k = 0; k < TAresultLists.Count; k++)
                                        if (overallScore > TAresultLists[k].score)
                                        {
                                            added = true;
                                            TAresultLists.Insert(k, newItem);
                                            break;
                                        }
                                    // add to the result lsit
                                    if (!added)
                                        TAresultLists.Insert(k, newItem);

                                    // update minimum score
                                    TAminScore = TAresultLists[TAresultLists.Count - 1].score;
                                }

                                zScore = ((double)TAminScore - inUsedMean) / inUsedStandarDeviation;
                                cumulativeProb = normDistribution.CumulativeDistribution(zScore);
                                righttailProb = 1 - cumulativeProb;

                                if (showStat)
                                {
                                    labelCIZScore.Text = String.Format("{0}", zScore);
                                    labelCIZScore.Refresh();
                                    labelCICumulativeProbability.Text = String.Format("{0}", cumulativeProb * 100);
                                    labelCICumulativeProbability.Refresh();
                                    labelCIRighttailProbability.Text = String.Format("{0}", righttailProb * 100);
                                    labelCIRighttailProbability.Refresh();
                                }
                            }
                        }

                        if (showStat)
                        {
                            labelCICurrentMinScore.Text = TAminScore.ToString();
                            labelCICurrentMinScore.Refresh();
                        }

                        if (roundUpTo > 0)
                        {
                            if ((Math.Round(cumulativeProb * 100, roundUpTo) >= givenProb) && (TAresultLists.Count == valueOfK))
                            {
                                stop = true;
                                break;
                            }
                        }
                        else
                        {
                            if ((cumulativeProb * 100 >= givenProb) && (TAresultLists.Count == valueOfK))
                            {
                                stop = true;
                                break;
                            }
                        }
                    }
                }
                else stop = true;

                if (stop)
                    break;
            }
            TimeSpan totalProbTATime = DateTime.Now - startTime;

            if (checkBoxSaveCIResultToFile.Checked)
            {
                StreamWriter swProbTAResult = new StreamWriter("C:\\Users\\Khoa\\Documents\\Visual Studio 2010\\Projects\\Zipf_Data\\" + textBoxSaveCIResultToFile.Text + ".probta.sorted.result.txt");
                swProbTAResult.WriteLine(numberOfList + " - Number of list");
                swProbTAResult.WriteLine(numberOfItem + " - Number of item per list");
                swProbTAResult.WriteLine("Zipf - Distribution of each list");
                swProbTAResult.WriteLine(sZipf + " - s parameter");
                swProbTAResult.WriteLine(valueOfK + " - Value of K");
                swProbTAResult.WriteLine("Sorted - Type of data");
                swProbTAResult.WriteLine("Traditional TA in Combination with Probability - Method");
                if (radioButtonCalculatePopParams.Checked)
                {
                    swProbTAResult.WriteLine("Calculate Population Mean and Population Standard Deviation");
                    swProbTAResult.WriteLine(caluclatingParamsTime.ToString() + " - Calculating Population Mean and Population Standard Deviation time");
                }
                else if (radioButtonGivenPopulationParams.Checked)
                {
                    swProbTAResult.WriteLine("Use GIVEN population Mean and population Standard Deviation");
                }
                else 
                {
                    swProbTAResult.WriteLine("Using Sample Mean and Sample Standard Deviation - Method");
                }
                swProbTAResult.WriteLine(confidence + " - Confidence");
                swProbTAResult.WriteLine(givenProb + " - Calculated Cumulative Probability");
                swProbTAResult.WriteLine(cumulativeProb + " - Cumulative Probability");
                swProbTAResult.WriteLine(numberOfItemHaveBeenRead.ToString() + "/" + (numberOfItem * numberOfList).ToString() + " - Number of data read/Total data");
                swProbTAResult.WriteLine(numberOfItemHaveBeenSeen.ToString() + "/" + numberOfItem.ToString() + " - Number of items seen/Total item");
                swProbTAResult.WriteLine(totalProbTATime.ToString() + " - Running Probability TA time");
                swProbTAResult.WriteLine((totalProbTATime + caluclatingParamsTime).ToString() + " - Total running time");
                swProbTAResult.WriteLine("---------------------------------------");
                for (i = 0; i < valueOfK; i++)
                    swProbTAResult.WriteLine(String.Format("{0,8}{1,15}", TAresultLists[i].index, TAresultLists[i].score));

                swProbTAResult.Close();
            }

            labelCIStatus.ForeColor = Color.Green;
            labelCIStatus.Text = "DONE";
            labelCIStatus.Refresh();
        }
        #endregion

        /// <summary>
        /// This function calculates the actual needed cumulative probability 
        /// </summary>
        /// <param name="inputConfidence"></param>
        /// <returns></returns>
        private double calculateProbTAConfidence(double inputConfidence)
        {
            double result = 0;
            double doubleExtra = 0;
            int extraElement = 0;
            double doubleOneOrZero = 0;

            if (inputConfidence < 100.0)
            {
                doubleExtra = 100 - inputConfidence;
                doubleExtra /= 100;
                double doubleUnFloorValue = doubleExtra * (double)valueOfK;
                double doubleFloorValue = Math.Floor(doubleUnFloorValue);
                if (doubleUnFloorValue - doubleFloorValue > 0)
                    doubleOneOrZero = 1.0;
                extraElement = Convert.ToInt32(doubleFloorValue + doubleOneOrZero);
            }

            result = 100 - ((((double)valueOfK + (double)extraElement) / (double)numberOfItem) * 100.0);

            return result;
        }

        private void radioButtonCalulatePopParams_CheckedChanged(object sender, EventArgs e)
        {
            if (radioButtonCalculatePopParams.Checked)
            {
                radioButtonSampleParams.Checked = false;
                radioButtonGivenPopulationParams.Checked = false;
            }
        }

        private void radioButtonGivenPopulationParams_CheckedChanged(object sender, EventArgs e)
        {
            if (radioButtonGivenPopulationParams.Checked)
            {
                radioButtonCalculatePopParams.Checked = false;
                radioButtonSampleParams.Checked = false;
            }
        }

        private void radioButtonSampleParams_CheckedChanged(object sender, EventArgs e)
        {
            if (radioButtonSampleParams.Checked)
            {
                radioButtonCalculatePopParams.Checked = false;
                radioButtonGivenPopulationParams.Checked = false;
            }
        }

        private void checkBoxShowState_CheckedChanged(object sender, EventArgs e)
        {
            if (checkBoxShowState.Checked)
                showStat = true;
            else showStat = false;
        }

        #region Sample Population Mean and Population Standard Deviation
        private void buttonProbTASampleParameters_Click(object sender, EventArgs e)
        {
            labelCIStatus.ForeColor = Color.Green;

            Random rand = new Random(DateTime.Now.Millisecond + DateTime.Now.Second + DateTime.Now.Minute + DateTime.Now.Hour
                                   + DateTime.Now.Day + DateTime.Now.Month + DateTime.Now.Year);

            #region Estimate Standard Deviation
            labelCIStatus.Text = "Estimating Standard Deviation...";
            labelCIStatus.Refresh();

            int tempN = Convert.ToInt32(textBoxProbTATempN.Text);
            int i;
            int[] randomPick = new int[tempN];
            for (i = 0; i < tempN; i++)
            {
                int index = rand.Next(1, numberOfItem);
                randomPick[i] = totalScore(index);
            }

            double randomSum = 0;
            for (i = 0; i < tempN; i++)
                randomSum += (double)randomPick[i];

            double randomMean = randomSum / (double)tempN;

            double randomSquareSum = 0;
            for (i = 0; i < tempN; i++)
            {
                double currentPick = (double)randomPick[i];
                randomSquareSum += (currentPick - randomMean) * (currentPick - randomMean);
            }

            estimatedStandardDeviation = Math.Sqrt(randomSquareSum / (double) tempN);

            textBoxProbTAEstimatedStandardDeviation.Text = String.Format("{0}", estimatedStandardDeviation);
            textBoxProbTAEstimatedStandardDeviation.Refresh();
            #endregion

            #region Calculate Sample Size
            labelCIStatus.Text = "Loading Math library...";
            labelCIStatus.Refresh();

            Normal normDistribution = new Normal();
            StudentT studentTDistribution = new StudentT();

            labelCIStatus.Text = "Calculating sample size...";
            labelCIStatus.Refresh();

            double marginOfError = Convert.ToDouble(textBoxProbTAMarginOfError.Text);
            double halfAlpha = (1.0 - Convert.ToDouble(textBoxProbTAConfidenceInterval.Text) / 100.0) / 2.0;
            double zScore = normDistribution.InverseCumulativeDistribution(1 - halfAlpha);

            double doubleN = (zScore * estimatedStandardDeviation / marginOfError) * (zScore * estimatedStandardDeviation / marginOfError);
            int intN = (int)Math.Ceiling(doubleN);

            studentTDistribution.DegreesOfFreedom = intN - 1;
            studentTDistribution.CumulativeDistribution(halfAlpha);
            #endregion

            labelCIStatus.Text = "DONE";
            labelCIStatus.Refresh();
        }
        #endregion

        #region Generate Data with Normal Distribution and Save to File
        private void buttonNormalGenerate_Click(object sender, EventArgs e)
        {
            labelStatusNormalGenerate.ForeColor = Color.Green;
            labelStatusNormalGenerate.Text = "Running...";
            labelStatusNormalGenerate.Refresh();

            numberOfList = Convert.ToInt32(textBoxNumberOfList.Text);
            numberOfItem = Convert.ToInt32(textBoxNumberOfItem.Text);

            int i, j;
            var normalData = new int[numberOfItem];
            double genMean = Convert.ToDouble(textBoxNormalGenerateMean.Text);
            double genStandardDeviation = Convert.ToDouble(textBoxNormalGenerateStandardDeviation.Text);

            StreamWriter swDataFile = new StreamWriter(strDataPath + textBoxSaveData.Text);
            swDataFile.WriteLine("{0}", numberOfList);
            swDataFile.WriteLine("{0}", numberOfItem);
            swDataFile.WriteLine("{0:0.00000}\t{1:0.00000}", genMean, genStandardDeviation);
            //swDataFile.WriteLine("...");
            for (i = 0; i < numberOfList; i++)
            {
                swDataFile.WriteLine("--------------------------------------------------");
                var normalRNG = new Normal(genMean, genStandardDeviation);
                int seed = DateTime.Now.Year + DateTime.Now.Month + DateTime.Now.Day
                         + DateTime.Now.Hour + DateTime.Now.Minute + DateTime.Now.Second
                         + DateTime.Now.Millisecond + i;
                normalRNG.RandomSource = new Random(seed);
                Random rand = new Random(seed);
                for (j = 0; j < numberOfItem; j++)
                {
                    double value = normalRNG.Sample();
                    //int intVal = rand.Next(10000);
                    int intVal = (int)value;
                    //if (value - (double)(intVal) >= 0.5)
                        //intVal++;
                    normalData[j] = intVal;
                    swDataFile.WriteLine(normalData[j]);
                }
            }
            swDataFile.Close();

            labelStatusNormalGenerate.ForeColor = Color.Green;
            labelStatusNormalGenerate.Text = "DONE";
            labelStatusNormalGenerate.Refresh();
        }
        #endregion

        #region Generate Data with default generator and Save to File
        private void buttonMinMaxGenerate_Click(object sender, EventArgs e)
        {
            labelStatusMinMaxGenerate.ForeColor = Color.Green;
            labelStatusMinMaxGenerate.Text = "Running...";
            labelStatusMinMaxGenerate.Refresh();

            numberOfList = Convert.ToInt32(textBoxNumberOfList.Text);
            numberOfItem = Convert.ToInt32(textBoxNumberOfItem.Text);

            int i, j;
            var minMaxData = new int[numberOfItem];

            StreamWriter swDataFile = new StreamWriter(strDataPath + textBoxSaveData.Text);
            swDataFile.WriteLine("{0}", numberOfList);
            swDataFile.WriteLine("{0}", numberOfItem);
            swDataFile.WriteLine("...");
            for (i = 0; i < numberOfList; i++)
            {
                swDataFile.WriteLine("--------------------------------------------------");
                int seed = DateTime.Now.Year + DateTime.Now.Month + DateTime.Now.Day
                         + DateTime.Now.Hour + DateTime.Now.Minute + DateTime.Now.Second
                         + DateTime.Now.Millisecond + i;
                Random rand = new Random(seed);
                /*int minVal = rand.Next(5000);
                int maxVal = rand.Next(5000);
                if (maxVal < minVal)
                {
                    int temp;
                    temp = minVal;
                    minVal = maxVal;
                    maxVal = temp;
                }*/
                for (j = 0; j < numberOfItem; j++)
                {
                    int intVal = rand.Next(10000);
                    minMaxData[j] = intVal;
                    swDataFile.WriteLine(minMaxData[j]);
                }
            }
            swDataFile.Close();

            labelStatusMinMaxGenerate.ForeColor = Color.Green;
            labelStatusMinMaxGenerate.Text = "DONE";
            labelStatusMinMaxGenerate.Refresh();
        }
        #endregion

        #region Test R
        private void buttonTestR_Click(object sender, EventArgs e)
        {
            //int dataSize = 100;
            //rconn.SetSymbol("sdataSize", dataSize);

            ////Generate Data in R
            //rconn.Evaluate("sdata<-rnorm(sdataSize)");
            //double[] data = (double[])rconn.GetSymbol("sdata");


            ////Calculate Statistics in R
            //rconn.Evaluate("saverage<-mean(sdata)");
            //rconn.Evaluate("sstdev<-sd(sdata)");
            //double average = (double)rconn.GetSymbol("saverage");
            //double stdev = (double)rconn.GetSymbol("sstdev");

            //// Display Data and Results on Windows Form
            //lbDataList.Items.Clear();
            //for (int c = 0; c < dataSize; c++)
            //{
            //    lbDataList.Items.Add(data[c]);
            //}
            //lblAverage.Text = "Average: " + average;
            //lblStdDev.Text = "Standard Deviation: " + stdev;

            //REngine.SetDllDirectory(@"C:\Program Files\R\R-2.15.0\bin\i386");
            //var engine = REngine.CreateInstance("RDotNet"); 

            /*
            var x = engine.CreateNumericVector(Enumerable.Range(0, 13).Select(i => i * Math.PI / 12).ToArray());
            var cos = engine.GetSymbol("cos").AsFunction();
            var y = cos.Invoke(new[] { x }).AsNumeric();
            Console.WriteLine(string.Join(" ", y));
            */
            //NumericVector group1 = engine.CreateNumericVector(new double[] {1, 2, 3, 4});
            //engine.SetSymbol("x", group1);
            ////GenericVector result = engine.EagerEvaluate("shapiro.test(x)").AsList();
            //GenericVector result = engine.EagerEvaluate("y <- c(1, 2, 3, 4)").AsList();

            int i;
            DateTime now = DateTime.Now;
            Random rand = new Random(now.Second + now.Minute + now.Hour + now.Day + now.Month + now.Year);
            int intSampleSize = rand.Next(3, 5000);
            double[] doubleSumList = new double[intSampleSize];
            for (i = 0; i < intSampleSize; i++)
            {
                int j = rand.Next(0, numberOfItem - 1);
                doubleSumList[i] = sumList[j];
            }
            NumericalVariable x = new NumericalVariable(doubleSumList);
            ShapiroWilkTest sTest = new ShapiroWilkTest(x);
            double a = sTest.GetPValue(HypothesisType.OneTailedUpper);

            labelSampleSize.Text = intSampleSize.ToString();
            labelPValue.ForeColor = Color.Green;
            labelPValue.Text = String.Format("{0:0.#####}", a);
        }
        #endregion
    }
}
